
from numpy import *

def loadDataSet():
    postingList = [['my', 'dog', 'has', 'flea',
                    'problems', 'help', 'please'],
                   ['maybe','not','take','hime',
                    'to','dog', 'park', 'stupid'],
                   ['my','dalmation','is', 'so', 'cute', 'I', 'love', 'him'],
                   ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                   ['mr', 'licks', 'ate', 'my', 'steak', 'how',
                    'to', 'stop', 'him'],
                   ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
    #上述单词组是否为侮辱性单词组,0不是1是
    classVec = [0,1,0,1,0,1]
    return postingList, classVec

'''
    创建词汇集合（去重后）
'''
def createVocabList(dataSet):
    vocabSet = set([])
    for document in dataSet:
        vocabSet = vocabSet | set(document)  #两个集合的并集
    return list(vocabSet)

'''
    测试inputSet，在vocabList中是否存在,并生成等长向量     
    :param vocabList 词汇表
    :param inputSet 待检测词汇表
    :return 返回与vocabList等长的向量(1表示存在,0表示不存在)
'''
def setOfWords2Vec(vocabList, inputSet):
    returnVec = [0] * len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] = 1
        else: print("the word: %s is not in my Vocabulary" % word)
    return returnVec

'''
    与setofWords2Vec类型，此函数为词袋模型
    不同之处在于，setOfWords2Vec函数生成vocabList相等长度的向量，值1表示inputSet中该单词存在,即=1
    本函数bagOfWords2VecMN,生成的vocabList同长向量，值是inputSet中单词出现的数量 即+=1
'''
def bagOfWords2VecMN(vocabList, inputSet):
    returnVec = [0] * len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] += 1
    return returnVec

'''
    朴素贝叶斯分类器训练函数:返回两个类别各词在本组中的比例
    :param trainMatrix  文档矩阵
    :param trainCategory    每篇文档类别标签所构成的向量
    :returns 
        :return p0Vec 非侮辱性词组，各词的占比例，数组
        :return p1Vec 侮辱性词组，各词的占比例，数组
        :return pAbusive 侮辱性词组整体占比(例子中为3/6 = 0.5)  [0,1,0,1,0,1]
'''
def trainNB0(trainMatrix, trainCategory):
    #词组数:6组
    numTrainDocs = len(trainMatrix)
    #第一篇文章的单词数存在与否向量
    numWords = len(trainMatrix[0])
    #侮辱类词组的数量/词组数量 ,即计算总侮辱类词组的比例
    pAbusive = sum(trainCategory) / float(numTrainDocs)

    '''
    p0Num = zeros(numWords)
    p1Num = zeros(numWords)

    p0Denom = 0.0
    p1Denom = 0.0
    '''
    p0Num = ones(numWords)
    p1Num = ones(numWords)

    p0Denom = 2.0
    p1Denom = 2.0

    for i in range(numTrainDocs):
        if trainCategory[i] == 1:
            #每一组类别为1的单词组中的单词+1
            p1Num += trainMatrix[i]
            #类别1单词总数+本单词组总数
            p1Denom += sum(trainMatrix[i])
        else:
            p0Num += trainMatrix[i]
            p0Denom += sum(trainMatrix[i])
    # print(p1Num, p1Denom)
    # print(p0Num, p0Denom)
    '''
    p1Vect = p1Num / p1Denom
    p0Vect = p0Num / p0Denom
    '''
    #使用log函数为了防止数据过小，近似0后适成的错误
    p1Vect = log(p1Num / p1Denom)
    p0Vect = log(p0Num / p0Denom)
    return p0Vect, p1Vect, pAbusive

'''
   通过深度学习分类
'''
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
    # 计算待测向量分别与两个类别的概率相乘，然后向量中概率求和与类别的对数概率相加
    p1 = sum(vec2Classify * p1Vec) + log(pClass1)
    p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)

    # 取概率较大概率者对应的类别分类
    if p1 > p0:
        return 1
    else:
        return 0

'''
    初始化训练样本数据
'''
def initTrain():


    # 加载DataSet
    postList, listClasses = loadDataSet()
    myVocabList = createVocabList(postList)
    trainMat = []
    # 计算每个单词组在总单词组中出现的向量表示
    for postinDoc in postList:
        trainMat.append(setOfWords2Vec(myVocabList, postinDoc))
    return trainMat, myVocabList, listClasses


'''
    测试数据
'''
def testingNB():
    trainMat, myVocabList, listClasses = initTrain()
    p0V, p1V, pAb = trainNB0(trainMat, listClasses)
    testEntry = ['love', 'my', 'dalmation']
    thisDoc = array(bagOfWords2VecMN(myVocabList, testEntry))
    #thisDoc = array(setOfWords2Vec(myVocabList, testEntry))
    print(testEntry, 'classifyed as : ', classifyNB(thisDoc, p0V, p1V, pAb))
    testEntry = ['stupid', 'garbage']
    thisDoc = array(bagOfWords2VecMN(myVocabList, testEntry))
    #thisDoc = array(setOfWords2Vec(myVocabList, testEntry))
    print(testEntry, 'classified as :', classifyNB(thisDoc, p0V, p1V, pAb))


def test():
    # 加载DataSet
    postList, listClasses = loadDataSet()
    myVocabList = createVocabList(postList)
    trainMat = []
    # 计算每个单词组在总单词组中出现的向量表示
    for postinDoc in postList:
        trainMat.append(setOfWords2Vec(myVocabList, postinDoc))

    p0V, p1V, pAb = trainNB0(trainMat, listClasses)
    print(p0V)
    print(p1V)
    print(pAb)

def textParse(bigString):
    import re
    # mySent = "This book is the best book on python or M.L. I have erver laid eyes upon."
    # mySend.split()
    # regEx = re.compile('\\W*')
    listOfTokens = re.split(r'\W+', bigString)
    tokens = [tok.lower() for tok in listOfTokens if len(tok) > 2]
    return tokens

'''
    垃圾邮件过滤，样本训练+测试
'''
def spamTest():
    docList = []
    classList = []
    # fullText = []

    for i in range(1, 26):
        # 构建spam单词组集
        wordList = textParse(open('email/spam/%d.txt' % i).read())
        docList.append(wordList)
        # fullText.extend(wordList)
        classList.append(1)

        # 构建ham单词组集
        wordList = textParse(open('email/ham/%d.txt' % i).read())
        docList.append(wordList)
        # fullText.extend(wordList)
        classList.append(0)

    # 构建不重复的唯一单词总集
    vocabList = createVocabList(docList)
    trainingSet = list(range(50))
    testSet = []

    # 生成10个随机数
    for i in range(10):
        randIndex = int(random.uniform(0, len(trainingSet)))
        testSet.append(trainingSet[randIndex])
        del(trainingSet[randIndex])

    #
    trainMat = []
    trainClasses = []

    #
    for docIndex in trainingSet:
        trainMat.append(setOfWords2Vec(vocabList, docList[docIndex]))
        trainClasses.append(classList[docIndex])

    p0V, p1V, pSpam = trainNB0(array(trainMat), array(trainClasses))
    errorCount = 0

    for docIndex in testSet:
        wordVector = setOfWords2Vec(vocabList, docList[docIndex])
        myClass = classifyNB(array(wordVector), p0V, p1V, pSpam)
        if myClass != classList[docIndex]:
            print('myClass is %d, but realClass is %d' %(myClass, classList[docIndex]))
            errorCount += 1

    print('the error rate is:%.2f%%' % float(errorCount * 100 / len(testSet)))


def calcMostFreq(vocabList, fullText):
    import operator
    freqDict = {}
    for token in vocabList:
        freqDict[token] = fullText.count(token)
    sortedFreq = sorted(freqDict.items(), key=operator.itemgetter(1), reverse=True)
    return sortedFreq[:30]

def localWords(feed1, feed0):
    # import feedparser
    docList = []
    classList = []
    fullText = []
    minLen = min(len(feed1['entries']), len(feed0['entries']))
    for i in range(minLen):
        wordList = textParse(feed1['entries'][i]['summary'])
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(1)
        wordList = textParse(feed0['entries'][i]['summary'])
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(0)
    vocabList = createVocabList(docList)
    top30Words = calcMostFreq(vocabList, fullText)

    for pairW in top30Words:
        if pairW[0] in vocabList: vocabList.remove(pairW[0])
    trainingSet = list(range(2*minLen))
    testSet = []
    for i in range(20):
        randIndex = int(random.uniform(0,len(trainingSet)))
        testSet.append(trainingSet[randIndex])
        del(trainingSet[randIndex])
    trainMat = []
    trainClasses = []
    for docIndex in trainingSet:
        trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
        trainClasses.append(classList[docIndex])
    p0V, p1V, pSpam = trainNB0(array(trainMat), array(trainClasses))
    errorCount = 0
    for docIndex in testSet:
        wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
        if classifyNB(array(wordVector), p0V, p1V, pSpam) != classList[docIndex]:
            errorCount += 1
    print("the error rate is : ", float(errorCount)/len(testSet))
    return vocabList, p0V, p1V

